
thresh = read_csv("./replication_file/_4_data/_thresholds_update.csv") 
pop <- read_csv('./replication_file/_4_data/census_stack_long_new_bc.csv') %>%
  group_by(city, YEAR) %>%
  summarise(pop=sum(num))

civil_reform = 
  read_csv("./replication_file/_4_data/cities_reform.csv") 

state_abbrev = 
  read_csv("./replication_file/_4_data/states.csv") 

treat_time <- thresh %>%
  mutate(min_pop = as.numeric(str_remove_all(min_pop, ','))) %>% 
  filter(coverage=='total' & pop_threshold==1 & optional==0) %>%
  select(-city) %>%
  filter(str_detect(specific_dept, 'civil service') | is.na(specific_dept)) %>% 
  group_by(abbrev, state, min_pop, max_pop) %>%
  # minimum year
  summarise(rd_year = min(year)) %>%
  # adding in population and seeing if in threshold
  inner_join(pop %>%
               mutate(abbrev = str_trim(str_extract(city, '[a-z]{2}$')))) %>%
  mutate(time = rd_year < YEAR,
         treat = case_when(rd_year <= YEAR & pop >= min_pop & is.na(max_pop) ~ 1,
                           rd_year <= YEAR & is.na(min_pop) & pop <= max_pop ~ 1,
                           rd_year <= YEAR & pop >= min_pop & pop <= max_pop ~ 1,
                           T ~ 0),
         bandwidth = ifelse(!is.na(min_pop), pop - min_pop, max_pop - pop)) %>%
  #filter(city=='adams, ma') %>% View()
  group_by(city, YEAR) %>%
  filter(treat == max(treat)) %>%
  filter(abs(bandwidth)== min(abs(bandwidth))) %>%
  select(-min_pop, -max_pop) %>%
  rename(treat_assign = treat) %>%
  distinct() %>%
  # joining with existing civil service data to look at first stage
  full_join(civil_reform %>%
              left_join(state_abbrev) %>%
              mutate(city = paste0(str_to_lower(str_trim(City)), ', ', str_to_lower(str_trim(Abbreviation)))) %>%
              filter(!(police_only==0 & fire_only==1)) %>%
              group_by(city) %>%
              summarise(actual_year = min(Year))) %>%
  mutate(treat_actual = ifelse(YEAR >= actual_year, 1, 0))

iv_data <- analysis_df_3 %>%
  filter(occ %in% c('white_collar', 'blue_collar')) %>%
  pivot_longer(., cols = c(govt_black_x_foreign_born:govt_white_x_native_born, black_x_foreign_born:white_x_native_born)) %>%
  mutate(govt = case_when(str_detect(name, 'govt') ~ 'govt',
                          T ~ 'pop'),
         name = str_remove(name, 'govt_')) %>%
  pivot_wider(id_cols = c(city, YEAR, occ, name), names_from = govt, values_from = value) %>%
  bind_rows(analysis_df_4 %>%
              filter(occ %in% c('white_collar', 'blue_collar')) %>%
              pivot_longer(., cols = c(govt_german:govt_russian, german:russian)) %>%
              mutate(govt = case_when(str_detect(name, 'govt') ~ 'govt',
                                      T ~ 'pop'),
                     name = str_remove(name, 'govt_')) %>%
              pivot_wider(id_cols = c(city, YEAR, occ, name), names_from = govt, values_from = value)) %>%
  inner_join(treat_time %>% 
               select(city, YEAR, treat_assign, treat_actual, bandwidth)) %>%
  filter(!is.na(treat_assign) & !is.na(treat_actual)) 



#FIRST STAGE
fs_mod <-
  iv_data %>%
  dplyr::select(city, YEAR, treat_assign, treat_actual) %>%
  distinct() %>%
  lm(treat_actual ~ treat_assign + factor(YEAR) + factor(city), data = .)

rf_mod <-
  iv_data %>%
  group_by(occ, name) %>%
  do(model = lm(govt ~ treat_assign + pop + factor(YEAR) + factor(city), data = .))

iv_data <-
  iv_data %>%
  mutate(fitted = lm(treat_actual ~ treat_assign + pop + factor(YEAR) + factor(city), data = .)$fitted)

ss_mod <-
  iv_data %>%
  group_by(occ, name) %>%
  do(model = lm(govt ~ fitted + pop + factor(YEAR) + factor(city), data = .))

blue_collar_mods <-
  list(fs_mod,
       #second stage
       ss_mod$model[[12]], #white native
       ss_mod$model[[11]], #white foreign
       ss_mod$model[[2]], #black native
       ss_mod$model[[4]], #irish
       ss_mod$model[[3]], #german
       ss_mod$model[[5]], #italian
       ss_mod$model[[9]], #polish
       ss_mod$model[[10]]) #russian

rf_blue_collar_mods <-
  list(fs_mod,
       #reduced form
       rf_mod$model[[12]], #white native
       rf_mod$model[[11]], #white foreign
       rf_mod$model[[2]], #black native
       rf_mod$model[[4]], #irish
       rf_mod$model[[3]], #german
       rf_mod$model[[5]], #italian
       rf_mod$model[[9]], #polish
       rf_mod$model[[10]]) #russian)

  

table_bc =
  stargazer(blue_collar_mods,
            type = "latex",
            title = "Effect of Civil Service Reform on Representation",
            label = "tab:table_main",
            model.names = F,
            model.numbers = F,
            column.labels = c("Reform", "Race by Nativity", "Foreign-Born Whites"),
            multicolumn = T,
            column.separate = c(1, 3, 5),
            dep.var.labels = NULL,
            digits = 2,
            star.cutoffs = c(0.1, 0.05),
            keep = c("treat_assign", "fitted", "pop"),
            
            covariate.labels = c("Mandate", "$\\widehat{\\text{Reform}}$", "Population Share"),
            keep.stat = c("n")
  )

table_bc_rf =
  stargazer(rf_blue_collar_mods,
            type = "latex",
            title = "Effect of Civil Service Reform on Representation",
            label = "tab:table_main",
            model.names = F,
            model.numbers = F,
            column.labels = c("Reform", "Race by Nativity", "Foreign-Born Whites"),
            multicolumn = T,
            column.separate = c(1, 3, 5),
            dep.var.labels = NULL,
            digits = 2,
            star.cutoffs = c(0.1, 0.05),
            keep = c("treat_assign", "fitted", "pop"),
            
            covariate.labels = c("Mandate", "$\\widehat{\\text{Reform}}$", "Population Share"),
            keep.stat = c("n")
  )


note_text <- paste("Table shows the effect of the introduction of municipal civil service reform on group representation in different classes of employment in city government. 
                  Points represent regression estimates, standard errors calculated with robust standard errors. 
                    Regressions control for the share of group population in a given municipality.
                   Outcome data were derived from IPUMS census data.")

table_bc_print <-
  table_bc[-c(12, 17, 20, 23, 28)] %>%
  append(., c("\\cline{2-2} \\cline{3-5} \\cline{6-10}",
  " &  &  &  &  &  &  &  & & \\\\",
  "\\textbf{Panel A: Blue Collar} & --- & N. White & F. White & N. Black & Irish & Italian & German & Russian & Polish \\\\"), after = 12)

table_bc_print <-
  table_bc_print[-c(17, 18)] %>%
  append(., table_bc_rf[c(15, 16)], after = 16)

#write_latex_placebo(table_bc_print, note_text, './replication_file/_5_outputs/tables/iv_analysis_table.tex', scale = 0.8)
write_latex_placebo(table_bc_print, note_text, './replication_file/_5_outputs/tables/table_1.tex', scale = 0.8)



